# -*- coding: utf-8 -*-

## Copyright (c) 2008 CMScom 
## Manabu Terada

import re
import unicodedata

from Products.CMFPlone.utils import classImplements
from Products.ZCTextIndex.ISplitter import ISplitter
from Products.ZCTextIndex.PipelineFactory import element_factory

enc = 'utf-8'
STOP_WORD = []

## Setting, adding langs.
rangetable = dict(
    # ascii = u"a-zA-Z0-9_",
    digit = u"\d",
    
    # U+AC00-D7AF 	Hangul Syllables 	ハングル音節文字
    hangul = u"\uAC00-\uD7AF",
    
    # U+30A0-30FF 	Katakana 	片仮名
    # U+3040-309F 	Hiragana 	平仮名
    #kana = u"\u3040-\u30FF",
    hiragana = u"\u3040-\u309F",
    katakana = u"\u30A0-\u30FF",    
    
    # U+4E00-9FFF 	CJK Unified Ideographs 	CJK統合漢字
    # U+3400-4DBF 	CJK Unified Ideographs Extension A 	CJK統合漢字拡張A
    # U+F900-FAFF 	CJK Compatibility Ideographs 	CJK互換漢字
    ideo = u"\u4E00-\u9FFF\u3400-\u4DBF\uF900-\uFAFF",
    
    thai = u"\u0E00-\u0E7F", # U+0E00-0E7F Thai タイ文字
)

igns = (u"\s", u"\!-\&", u"\:-\@", u"\(-\/", u"\{-\~")
# igns = r"\W"
## End of setting.

## Splitting core.
ps = rangetable.values() + [u"^"+u"".join(rangetable.values() + list(igns))]

glob_true = u"|".join(u"[%s]+[*?]*" % x  for x in ps)
glob_false = u"|".join(u"[%s]+" % x  for x in ps)

# pattern = re.compile(u"[a-zA-Z0-9_]+|[\uac00-\ud7af]+|[\u4E00-\u9FFF\u3400-\u4dbf\uf900-\ufaff\u3040-\u30ff]+", re.UNICODE)
# pattern_g = re.compile(u"[a-zA-Z0-9_]+[*?]*|[\u4E00-\u9FFF\u3400-\u4dbf\uf900-\ufaff\u3040-\u30ff\uac00-\ud7af]+[*?]*", re.UNICODE)
pattern = re.compile(glob_false, re.UNICODE)
pattern_g = re.compile(glob_true, re.UNICODE)

KANA_MIN = ord(u"ア")
KANA_MAX = ord(u"ア") + 85
KANA_DIF = ord(u"あ") - ord(u"ア")
# KIGOS = frozenset(ord(x) for x in u"-ー")

def bigrams(xs, isglob=False):
    for x in xs:
        for y in bigrams_(x, isglob):
            yield y

def bigrams_(s, isglob):
    assert isinstance(s, unicode)
    normed = unicodedata.normalize('NFKC', s)
    ## return bisplitx((pattern_g if isglob else pattern).findall(normed) , isglob)
    return bisplitx((isglob and pattern_g or pattern).findall(normed) , isglob)

def bisplitx(xss, isglob):
    for xs in xss:
        if not xs:
            break
        if ord(xs[0]) <= 122:
            yield xs
        else:
            for x in bisplit(xs, isglob):
                yield kananormalize(x)
            ## for x in bisplit(kananormalize(xs), isglob):
            ##     yield x

def bisplit(xs, isglob):
    assert xs
    length = len(xs)
    if length == 1 and isglob:
        yield xs + "*"
        return
    ## for i in xrange(length - 1 if isglob else length):
    for i in xrange((not isglob and length != 2) and length or length - 1):        
        w = xs[i:i+2]
        yield w

def kananormalize(xs):
    assert isinstance(xs, unicode)
    return u"".join(
        ## unichr(ord(x) + KANA_DIF) if KANA_MIN <= ord(x) <= KANA_MAX else x
        (KANA_MIN <= ord(x) <= KANA_MAX) and unichr(ord(x) + KANA_DIF) or x
        for x in xs)
## End of Splitting core.


## Main of Splitter.
def to_unicode(s):
    """ str to unicode, replace to stop word. """
    if isinstance(s, str):
        try:
            s = s.decode(enc)
        except (UnicodeDecodeError, TypeError):
            pass
    t = u"".join(t for t in s if t not in STOP_WORD)
    return t

class Splitter(object):
    """
    """
    meta_type = 'BigramSplitter'
    __implements__ = ISplitter

    def process(self, lst, glob=0):
        ulst = (to_unicode(t) for t in lst)
        return [t.encode(enc, 'replace') for t in bigrams(ulst, glob)]

    def processGlob(self, lst):
        return self.process(lst, 1)

classImplements(Splitter, Splitter.__implements__)

try:
    element_factory.registerFactory('Word Splitter',
        'Unicode Whitespace splitter', Splitter)
except ValueError:
    # In case the splitter is already registered, ValueError is raised
    pass


class CaseNormalizer(object):

    def process(self, lst):
        result = []
        for s in lst:
            # This is a hack to get the normalizer working with
            # non-unicode text.
            try:
                if not isinstance(s, unicode):
                    s = unicode(s, enc)
            except (UnicodeDecodeError, TypeError):
                result.append(s.lower())
            else:
                result.append(s.lower().encode(enc))
        return result

try:
    element_factory.registerFactory('Case Normalizer',
        'Unicode Case Normalizer', CaseNormalizer)
except ValueError:
    # In case the normalizer is already registered, ValueError is raised
    pass
## End of Spritter.
